{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import joblib"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 二级stacking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection import train_test_split\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "# 导入训练数据集\n",
    "data_date = pd.read_csv('../../preprocess_data/train_x_date.csv').drop(columns=['id','loan_hour'])\n",
    "data_raw = pd.read_csv('../../preprocess_data_new/train_ax_nodup.csv',nrows=33465).drop(columns=['id','loan_dt','tag'])\n",
    "# data_raw = pd.read_csv('../../preprocess_data/train_x_33465.csv').drop(columns=['id','loan_dt','tag'])\n",
    "# data_null = pd.read_csv('../../preprocess_data/train_x_null.csv').drop(columns=['id'])\n",
    "data_null = pd.read_csv('../../preprocess_data_new/train_ax_row_null.csv',nrows=33465).drop(columns=['id'])\n",
    "data_tag = pd.read_csv('../../preprocess_data/train_x_33465.csv',usecols=['tag'])\n",
    "\n",
    "data = pd.concat([data_date,data_raw,data_null,data_tag],axis=1)\n",
    "data_label = pd.read_csv('../../preprocess_data/train_y_33465.csv',usecols=['label'])\n",
    "x = data.fillna(-1).values\n",
    "y = data_label.values.ravel()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 不同用途的数据集产生（3种）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1.本地验证\n",
    "x_train, x_test, y_train, y_test = train_test_split(x,y,random_state=2018,test_size=0.3)\n",
    "\n",
    "# 2. 过采样\n",
    "# from imblearn.over_sampling import BorderlineSMOTE\n",
    "# x_resampled, y_resampled = BorderlineSMOTE().fit_resample(x_train, y_train)\n",
    "\n",
    "# 线上测试\n",
    "# x_train = x\n",
    "# y_train = y\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [],
   "source": [
    "def SelectModel(model_name):\n",
    "    if model_name == 'GBC':\n",
    "        from sklearn.ensemble import GradientBoostingClassifier\n",
    "        model = GradientBoostingClassifier(loss='deviance',\n",
    "                                           learning_rate =0.1,\n",
    "                                           n_estimators=300,\n",
    "                                           subsample=0.9,\n",
    "                                           max_depth=3,\n",
    "#                                            verbose=1,\n",
    "                                          random_state=2018)\n",
    "    elif model_name == 'XGB':\n",
    "        from xgboost import XGBClassifier\n",
    "\n",
    "        model = XGBClassifier(max_depth=6,\n",
    "                              learning_rate =0.04, \n",
    "                              booster='gbtree',\n",
    "                              objective='binary:logistic',\n",
    "                              early_stopping_rounds=100,\n",
    "                              scale_pos_weight=float(len(y_train)-np.sum(y_train))/float(np.sum(y_train)),\n",
    "                              eval_metric='auc',\n",
    "                              gamma=1,\n",
    "                              reg_lambda=1,\n",
    "                              subsample=0.9,\n",
    "                              min_child_weight=1,\n",
    "                              seed=2018,\n",
    "                              silent=False,\n",
    "                              n_jobs=24,\n",
    "                              num_boost_round =300\n",
    "                             )\n",
    "    elif model_name == 'RFC':\n",
    "        from sklearn.ensemble import RandomForestClassifier\n",
    "        model = RandomForestClassifier(n_estimators=1500,\n",
    "                                       n_jobs =36,\n",
    "                                       max_features='sqrt',\n",
    "                                       class_weight='balanced',\n",
    "#                                        verbose =1,\n",
    "                                       random_state=2018)\n",
    "    elif model_name == 'LGB':\n",
    "        from lightgbm import LGBMClassifier\n",
    "        model = LGBMClassifier(boost='gbdt',\n",
    "                    num_leaves=135, \n",
    "                    scale_pos_weight=float(len(y_train)-np.sum(y_train.ravel()))/float(np.sum(y_train.ravel())),\n",
    "                    max_depth=-1,\n",
    "                    learning_rate=.05,\n",
    "                    max_bin=200,\n",
    "                    min_data_in_leaf= 60,\n",
    "                    objective='binary',\n",
    "                    metric='auc',\n",
    "                    num_threads=32,\n",
    "                    slient=False,\n",
    "                    num_boost_round =300)\n",
    "    else:\n",
    "        pass\n",
    "    return model\n",
    "\n",
    "def get_oof(clf,n_folds,x_train,y_train,x_test):\n",
    "    '''\n",
    "    描述：\n",
    "        stacking的关键代码，通过k折法训练及预测训练集，避免data leak。\n",
    "    参数：\n",
    "        input：\n",
    "            clf: sklearn接口的学习器对象\n",
    "            其他：略\n",
    "    '''\n",
    "    n_train = x_train.shape[0]\n",
    "    n_test = x_test.shape[0]\n",
    "    num_class = len(np.unique(y_train))\n",
    "    kf = KFold(n_splits=n_folds,random_state=2018)\n",
    "    oof_train = np.zeros((n_train,num_class-1))\n",
    "    oof_test = np.zeros((n_test,num_class-1))\n",
    "    \n",
    "    for i,(train_index,test_index)in enumerate(kf.split(x_train,y_train)):\n",
    "        print(i)\n",
    "        kf_x_train = x_train[train_index]\n",
    "        kf_y_train = y_train[train_index]\n",
    "        kf_x_test = x_train[test_index]\n",
    "        \n",
    "        clf.fit(kf_x_train,kf_y_train)\n",
    "        tmp_trafold = clf.predict_proba(kf_x_test)\n",
    "        tmp_test = clf.predict_proba(x_test)\n",
    "        if tmp_trafold.ndim != 1:\n",
    "            tmp_trafold = tmp_trafold[:,1:]\n",
    "            tmp_test = tmp_test[:,1:]\n",
    "        oof_train[test_index] = tmp_trafold\n",
    "        oof_test += tmp_test\n",
    "        \n",
    "    oof_test = oof_test/n_folds\n",
    "    return oof_train,oof_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# stacking——第一级"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "# 1.训练第一级中的学习器\n",
    "model_list = ['XGB','RFC','LGB'] # GBC运行时间过长，所以未把它加入第一级\n",
    "new_train_list = []\n",
    "new_test_list = []\n",
    "for model in model_list:\n",
    "    clf = SelectModel(model)\n",
    "    oof_train,oof_test = get_oof(clf,5,x_train,y_train,x_test)\n",
    "    new_train_list.append(oof_train)\n",
    "    new_test_list.append(oof_test)\n",
    "# 2.整合第一级的输出作为第二级的输出\n",
    "new_train = np.hstack(tuple(new_train_list))\n",
    "new_test = np.hstack(tuple(new_test_list))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## stacking——第二级"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# 在第二级中，使用多个模型来融合第一层的输出，选出效果最佳的组合\n",
    "# lightgbm\n",
    "from lightgbm import LGBMClassifier\n",
    "lgb_model = LGBMClassifier(boost='gbdt',\n",
    "            num_leaves=145, \n",
    "            scale_pos_weight=float(len(y_train)-np.sum(y_train.ravel()))/float(np.sum(y_train.ravel())),\n",
    "            max_depth=3,\n",
    "            learning_rate=.01,\n",
    "            max_bin=200,\n",
    "            min_data_in_leaf= 3,\n",
    "            objective='binary',\n",
    "            metric='auc',\n",
    "            num_threads=32,\n",
    "            slient=False,\n",
    "            num_boost_round =120)\n",
    "lgb_model.fit(new_train,y_train)\n",
    "\n",
    "# random forest\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "rfc_model = RandomForestClassifier(n_estimators=800,\n",
    "                               n_jobs =36,\n",
    "                               criterion='gini',\n",
    "                               max_depth=5,\n",
    "                               min_samples_split=8,\n",
    "                               max_features='sqrt',\n",
    "                               class_weight='balanced',\n",
    "                               verbose =1,\n",
    "                               random_state=2018)\n",
    "rfc_model.fit(new_train,y_train)\n",
    "\n",
    "# xgboost\n",
    "from xgboost import XGBClassifier\n",
    "xgb_model = XGBClassifier(max_depth=3,\n",
    "                      learning_rate =0.02, \n",
    "                      booster='gbtree',\n",
    "                      objective='binary:logistic',\n",
    "                      early_stopping_rounds=100,\n",
    "                      scale_pos_weight=float(len(y_train)-np.sum(y_train))/float(np.sum(y_train)),\n",
    "                      eval_metric='auc',\n",
    "                      gamma=0.1,\n",
    "                      reg_lambda=1,\n",
    "                      subsample=0.9,\n",
    "                      min_child_weight=1,\n",
    "                      seed=2018,\n",
    "                      silent=False,\n",
    "                      n_jobs=24,\n",
    "                      num_boost_round =250\n",
    "                     )\n",
    "xgb_model.fit(new_train,y_train)\n",
    "\n",
    "# gradient boosting\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "gbc_model = GradientBoostingClassifier(loss='deviance',\n",
    "                                   learning_rate =0.05,\n",
    "                                   n_estimators=300,\n",
    "                                   subsample=0.9,\n",
    "                                   max_depth=3,\n",
    "                                   verbose=1,\n",
    "                                  random_state=2018)\n",
    "gbc_model.fit(new_train,y_train)\n",
    "\n",
    "# logistic regression\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "lg_model = LogisticRegression(penalty='l2',\n",
    "                        class_weight='balanced',\n",
    "                        random_state=2018,\n",
    "                        solver='liblinear',\n",
    "                        max_iter=150)\n",
    "lg_model.fit(new_train,y_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [],
   "source": [
    "# stacking第2层做blending,得到stack_pred\n",
    "y_pred = lgb_model.predict_proba(new_test)[:,1] + \\\n",
    "         rfc_model.predict_proba(new_test)[:,1] + \\\n",
    "         lgb_model.predict_proba(new_test)[:,1] + \\\n",
    "         3*gbc_model.predict_proba(new_test)[:,1] + \\\n",
    "         lg_model.predict_proba(new_test)[:,1]\n",
    "\n",
    "stack_pred = pd.DataFrame({'id':test_id})\n",
    "stack_pred['prob'] = y_pred\n",
    "stack_pred.to_csv('stack_pred.csv',index=False)\n",
    "\n",
    "# stacking第2层用单一模型，得到stack_pred1\n",
    "y_pred = lg_model.predict_proba(new_test)[:,1] \n",
    "stack_pred = pd.DataFrame({'id':test_id})\n",
    "stack_pred['prob'] = y_pred\n",
    "stack_pred.to_csv('stack_pred1.csv',index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "线上分数：  \n",
    "stack_pred AUC:0.8297  \n",
    "stack_pred1 AUC: 0.82757885054343"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
